# -*- coding: utf-8 -*-
"""
Created on Wed Jul 25 16:37:43 2018

@author: 君
"""

#文件去重
import os
#import json
#获取目标文件夹的路径（内含所有需要去重从文件）
filedir = os.getcwd()+'/data_major'
#获取当前文件夹中的文件名称列表
filenames=os.listdir(filedir)

for filename in filenames:
    filepath = filedir+'/'+filename
    ls = open(filepath).readlines()
    if len(ls) != 124:
        newls = {}
        file = open(filename, 'w', encoding='gbk')
        for i in range(len(ls)):
            if ls[i] in newls:
                continue
            else:
                newls[ls[i]] = 1
        a = list(newls.keys())
        for l in a:
            file.write(l)
        file.close()
        newls = {}
    else:
        continue
print(filepath+"去重成功！")